agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions 330+ messages / 2 participants [nested] [flat]
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions @ 2026-05-23 14:09 Mats Kindahl <mats@kindahl.net> 0 siblings, 0 replies; 330+ messages in thread From: Mats Kindahl @ 2026-05-23 14:09 UTC (permalink / raw) Two PostgreSQL standbys can independently promote to the same timeline ID if their primary stopped before either had a chance to promote. In that situation both clusters share a timeline history prefix that looks identical to pg_rewind: same TLI numbers and same begin/end LSNs. The existing same-TLI shortcut therefore treated the source as a valid rewind target and skipped the rewind entirely, leaving the target's diverged WAL intact. Fix this by embedding a UUIDv7 value in every timeline history file entry at promotion time. Each promotion generates a fresh UUID, so two independent promotions to the same TLI will carry different UUIDs even though the TLI number and begin LSN are identical. When loading the timeline history, pg_rewind uses these UUIDs in two places: 1. findCommonAncestorTimeline checks that the TLI and UUID in each entry match. A mismatch signals independent promotions and the search continues to earlier entries to find the true common ancestor. 2. The same-TLI shortcut (source and target on the same current TLI) compares the UUID stored in the last completed history entry and a mismatch forces a full rewind instead of a no-op. UUIDs are zero for clusters that predate this change, and the comparison function treats a zero UUID on either side as different from a UUID since that promotion has to be from a different server (it had a pre-change version server that was promoted, so it cannot be the same as a post-change version server that was promoted). Two new tests in t/005_same_timeline.pl cover both detection paths. The first covers the same-TLI shortcut: two standbys independently promote to TLI2 and TLI2', each with a distinct UUID. The second covers the ancestor search: the target goes through TLI1 -> TLI2 -> TLI3 while the source independently promoted so that it has a timeline with TLI1 -> TLI2' -> TLI3'. The test ensures that findCommonAncestorTimeline backs up to TLI1 as the true common ancestor rather than accepting the numerically matching TLI2 entry. --- src/backend/access/transam/timeline.c | 77 ++++- src/backend/access/transam/xlog.c | 15 + src/backend/utils/adt/uuid.c | 15 +- src/bin/pg_rewind/pg_rewind.c | 104 ++++++- src/bin/pg_rewind/t/005_same_timeline.pl | 362 +++++++++++++++++++++++ src/bin/pg_rewind/timeline.c | 47 ++- src/include/access/timeline.h | 5 +- src/include/access/xlog_internal.h | 1 + src/include/utils/uuid.h | 10 +- 9 files changed, 614 insertions(+), 22 deletions(-) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index d80c8ffe0a7..237511b7521 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -42,6 +42,8 @@ #include "pgstat.h" #include "storage/fd.h" #include "utils/wait_event.h" +#include "utils/fmgrprotos.h" +#include "utils/uuid.h" /* * Copies all timeline history files with id's between 'begin' and 'end' @@ -110,8 +112,12 @@ readTimeLineHistory(TimeLineID targetTLI) ereport(FATAL, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); - /* Not there, so assume no parents */ - entry = palloc_object(TimeLineHistoryEntry); + + /* + * Not there, so assume no parents. We use palloc0_object to ensure + * that tluuid is all-zero. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = entry->end = InvalidXLogRecPtr; return list_make1(entry); @@ -125,6 +131,7 @@ readTimeLineHistory(TimeLineID targetTLI) prevend = InvalidXLogRecPtr; for (;;) { + char uuid_str[UUID_STR_LEN + 1] = {0}; char fline[MAXPGPATH]; char *res; char *ptr; @@ -155,7 +162,8 @@ readTimeLineHistory(TimeLineID targetTLI) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = + sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -164,7 +172,7 @@ readTimeLineHistory(TimeLineID targetTLI) (errmsg("syntax error in history file: %s", fline), errhint("Expected a numeric timeline ID."))); } - if (nfields != 3) + if (nfields < 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), errhint("Expected a write-ahead log switchpoint location."))); @@ -176,12 +184,45 @@ readTimeLineHistory(TimeLineID targetTLI) lasttli = tli; - entry = palloc_object(TimeLineHistoryEntry); + /* + * We use palloc0_object to ensure that tluuid is all-zero, which is + * important for pg_rewind to detect whether the history file is + * missing or not. + */ + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = tli; entry->begin = prevend; entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4. It is in theory possible that the reason string + * starts with a UUID, but the current usage do not store a UUID. This + * allows us to support both old and new formats of history files + * without breaking compatibility by checking if the field contains a + * valid UUID. + */ + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + { + PG_TRY(); + { + Datum datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + memcpy(&entry->tluuid, DatumGetUUIDP(datum), sizeof(pg_uuid_t)); + } + PG_CATCH(); + { + ErrorData *edata = CopyErrorData(); + + FlushErrorState(); + ereport(FATAL, + errmsg("invalid UUID in history file \"%s\"", path), + errdetail("%s", edata->message)); + } + PG_END_TRY(); + } + /* Build list with newest item first */ result = lcons(entry, result); @@ -197,9 +238,11 @@ readTimeLineHistory(TimeLineID targetTLI) /* * Create one more entry for the "tip" of the timeline, which has no entry - * in the history file. + * in the history file. We use palloc0_object to ensure that tluuid is + * all-zero, which is important for pg_rewind to detect whether the + * history file is missing or not. */ - entry = palloc_object(TimeLineHistoryEntry); + entry = palloc0_object(TimeLineHistoryEntry); entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; @@ -294,21 +337,33 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent + * newTLUUID: UUID uniquely identifying this promotion instance * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * + * The output file is named <newTLI>.history (e.g. 00000003.history). If two + * servers independently promote to the same timeline ID, their history files + * share the same name. In a shared WAL archive the second file to arrive + * silently overwrites the first. The newTLUUID written into the file content + * lets pg_rewind detect this collision: it fetches each server's history file + * directly from that server, compares the UUIDs for every shared TLI, and + * treats a UUID mismatch as evidence of independent promotion even when the + * TLI numbers agree. + * * Currently this is only used at the end recovery, and so there are no locking * considerations. But we should be just as tense as XLogFileInit to avoid * emplacing a bogus file. */ void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason) { char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; char buffer[BLCKSZ]; + char *uuid_str; int srcfd; int fd; ssize_t nbytes; @@ -398,13 +453,19 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, * * If we did have a parent file, insert an extra newline just in case the * parent file failed to end with one. + * + * Format: <parentTLI>\t<switchpoint>\t<ThisTimeLineUUID>\t<reason>\n */ + uuid_str = DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(newTLUUID))); + snprintf(buffer, sizeof(buffer), - "%s%u\t%X/%08X\t%s\n", + "%s%u\t%X/%08X\t%s\t%s\n", (srcfd < 0) ? "" : "\n", parentTLI, LSN_FORMAT_ARGS(switchpoint), + uuid_str, reason); + pfree(uuid_str); nbytes = strlen(buffer); errno = 0; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..fde491bff5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,6 +99,7 @@ #include "storage/subsystems.h" #include "storage/sync.h" #include "utils/guc_hooks.h" +#include "utils/uuid.h" #include "utils/guc_tables.h" #include "utils/injection_point.h" #include "utils/pgstat_internal.h" @@ -6376,6 +6377,9 @@ StartupXLOG(void) newTLI = endOfRecoveryInfo->lastRecTLI; if (ArchiveRecoveryRequested) { + struct timeval tv; + pg_uuid_t uuid_buf; + newTLI = findNewestTimeLine(recoveryTargetTLI) + 1; ereport(LOG, (errmsg("selected new timeline ID: %u", newTLI))); @@ -6406,8 +6410,19 @@ StartupXLOG(void) * to the new timeline, and will try to connect to the new timeline. * To minimize the window for that, try to do as little as possible * between here and writing the end-of-recovery record. + * + * Generate a UUIDv7 that uniquely identifies this promotion. The + * same UUID is written into the history file so that pg_rewind can + * distinguish two servers that independently promoted to the same + * timeline ID. Use gettimeofday() since we are not on a hot path; + * generate_uuidv7 wants milliseconds and we pass 0 for sub-ms since + * the random bits already distinguish UUIDs generated within the same + * millisecond. */ + gettimeofday(&tv, NULL); + generate_uuidv7_r(&uuid_buf, tv.tv_sec * 1000 + tv.tv_usec / 1000, 0); writeTimeLineHistory(newTLI, recoveryTargetTLI, + &uuid_buf, EndOfLog, endOfRecoveryInfo->recoveryStopReason); ereport(LOG, diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index a9edceabab0..c153131e9f5 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -89,7 +89,7 @@ static bool uuid_abbrev_abort(int memtupcount, SortSupport ssup); static Datum uuid_abbrev_convert(Datum original, SortSupport ssup); static inline void uuid_set_version(pg_uuid_t *uuid, unsigned char version); static inline int64 get_real_time_ns_ascending(void); -static pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); Datum uuid_in(PG_FUNCTION_ARGS) @@ -616,6 +616,14 @@ get_real_time_ns_ascending(void) return ns; } +pg_uuid_t * +generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + + return generate_uuidv7_r(uuid, unix_ts_ms, sub_ms); +} + /* * Generate UUID version 7 per RFC 9562, with the given timestamp. * @@ -632,10 +640,9 @@ get_real_time_ns_ascending(void) * * NB: all numbers here are unsigned, unix_ts_ms cannot be negative per RFC. */ -static pg_uuid_t * -generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms) +pg_uuid_t * +generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms) { - pg_uuid_t *uuid = palloc(UUID_LEN); uint32 increased_clock_precision; /* Fill in time part */ diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..ffb12b1ba4a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -32,6 +32,19 @@ #include "rewind_source.h" #include "storage/bufpage.h" +/* + * Timeline histories for both clusters, populated by matchAndFetchTimelines(). + */ +typedef struct TimeLineHistoriesData +{ + TimeLineHistoryEntry *source, + *target; + int sourceNentries, + targetNentries; +} TimeLineHistoriesData; + +typedef TimeLineHistoriesData *TimeLineHistories; + static void usage(const char *progname); static void perform_rewind(filemap_t *filemap, rewind_source *source, @@ -53,6 +66,9 @@ static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, TimeLineHistoryEntry *b_history, int b_nentries, XLogRecPtr *recptr, int *tliIndex); +static inline bool matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b); +static bool matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, + TimeLineHistories timelineHistories); static void ensureCleanShutdown(const char *argv0); static void disconnect_atexit(void); @@ -141,6 +157,7 @@ main(int argc, char **argv) int c; XLogRecPtr divergerec; int lastcommontliIndex; + TimeLineHistoriesData timelineHistories; XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; @@ -374,10 +391,21 @@ main(int argc, char **argv) * * If both clusters are already on the same timeline, there's nothing to * do. + * + * This also handles the case when two servers independently promoted to + * the same timeline ID: one crashed after writing the history file but + * before its EOR WAL record was distributed, so a second standby promoted + * independently. The history files produced by those two promotions + * carry different UUIDs. + * + * When the clusters are on different timelines we locate the fork point + * via findCommonAncestorTimeline. */ - if (target_tli == source_tli) + if (matchAndFetchTimelines(source_tli, target_tli, &timelineHistories)) { pg_log_info("source and target cluster are on the same timeline"); + pfree(timelineHistories.source); + pfree(timelineHistories.target); rewind_needed = false; target_wal_endrec = InvalidXLogRecPtr; } @@ -391,8 +419,10 @@ main(int argc, char **argv) * Retrieve timelines for both source and target, and find the point * where they diverged. */ - sourceHistory = getTimelineHistory(source_tli, true, &sourceNentries); - targetHistory = getTimelineHistory(target_tli, false, &targetNentries); + targetHistory = timelineHistories.target; + targetNentries = timelineHistories.targetNentries; + sourceHistory = timelineHistories.source; + sourceNentries = timelineHistories.sourceNentries; findCommonAncestorTimeline(sourceHistory, sourceNentries, targetHistory, targetNentries, @@ -876,7 +906,7 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) */ if (tli == 1) { - history = pg_malloc_object(TimeLineHistoryEntry); + history = pg_malloc0_object(TimeLineHistoryEntry); history->tli = tli; history->begin = history->end = InvalidXLogRecPtr; *nentries = 1; @@ -922,6 +952,56 @@ getTimelineHistory(TimeLineID tli, bool is_source, int *nentries) return history; } +/* + * Return true if two per-entry promotion UUIDs are compatible. + * + * A zero UUID means the history file predates this fix (or the entry is + * synthetic). If both sides are zero we have no UUID information and fall + * back to TLI-number-only matching (backward compatibility with old servers). + * If one side carries a UUID and the other does not, they cannot originate + * from the same promotion and are treated as incompatible. + */ +static inline bool +matchingTimelineUUID(TimeLineHistoryEntry *a, TimeLineHistoryEntry *b) +{ + static const pg_uuid_t zero = {{0}}; + + if (memcmp(&a->tluuid, &zero, UUID_LEN) == 0 && memcmp(&b->tluuid, &zero, UUID_LEN) == 0) + return true; + return memcmp(&a->tluuid, &b->tluuid, UUID_LEN) == 0; +} + +/* + * Fetch the timeline history for both clusters, store them in tlh, and return + * true if the clusters are on the same timeline (no rewind needed). + * + * tlh is always fully populated on return regardless of the result, so the + * caller can pass tlh->source / tlh->target directly to + * findCommonAncestorTimeline() when the return value is false. + * + * TLI 1 always returns true: it is the original timeline and has no promotion + * UUID. For TLI >= 2, the UUID in entry[Nentries - 2] identifies the + * promotion that created the current TLI. Both-zero UUIDs (old history files) + * are treated as compatible; zero-vs-nonzero is treated as a mismatch because + * one side carries a promotion UUID and they cannot be the same promotion. + */ +static bool +matchAndFetchTimelines(TimeLineID source_tli, TimeLineID target_tli, TimeLineHistories tlh) +{ + tlh->source = getTimelineHistory(source_tli, true, &tlh->sourceNentries); + tlh->target = getTimelineHistory(target_tli, false, &tlh->targetNentries); + + if (source_tli != target_tli) + return false; + + /* TLI 1 has no promotion UUID; always treat as the same timeline. */ + if (tlh->sourceNentries < 2 || tlh->targetNentries < 2) + return true; + + return matchingTimelineUUID(&tlh->source[tlh->sourceNentries - 2], + &tlh->target[tlh->targetNentries - 2]); +} + /* * Determine the TLI of the last common timeline in the timeline history of * two clusters. *tliIndex is set to the index of last common timeline in @@ -943,12 +1023,26 @@ findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, int a_nentries, * depending on the history files that each node has fetched in previous * recovery processes. Hence check the start position of the new timeline * as well and move down by one extra timeline entry if they do not match. + * + * We also compare timeline UUIDs when both sides carry one. Two servers + * that independently promoted to the same timeline ID produce history + * files with the same name (e.g. 00000003.history); in a shared WAL + * archive the second file silently overwrites the first. pg_rewind + * fetches each server's history file directly from that server, so it + * sees both UUIDs. + * + * The timeline UUID stored in history entry[i] is the UUID of the + * promotion that created entry[i+1], i.e. the UUID of TLI entry[i+1].tli. + * So to check whether entry[i] itself represents the same timeline on + * both sides we look at entry[i-1].tluuid (for i > 0). TLI 1 (i == 0) is + * always the same: it is the original timeline and has no promotion UUID. */ n = Min(a_nentries, b_nentries); for (i = 0; i < n; i++) { if (a_history[i].tli != b_history[i].tli || - a_history[i].begin != b_history[i].begin) + a_history[i].begin != b_history[i].begin || + (i > 0 && !matchingTimelineUUID(&a_history[i - 1], &b_history[i - 1]))) break; } diff --git a/src/bin/pg_rewind/t/005_same_timeline.pl b/src/bin/pg_rewind/t/005_same_timeline.pl index 95a40c3b270..2360c3df1d0 100644 --- a/src/bin/pg_rewind/t/005_same_timeline.pl +++ b/src/bin/pg_rewind/t/005_same_timeline.pl @@ -7,6 +7,8 @@ # use strict; use warnings FATAL => 'all'; +use File::Copy; +use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -21,4 +23,364 @@ RewindTest::create_standby(); RewindTest::run_pg_rewind('local'); RewindTest::clean_rewind_test(); +# Helper function to run pg_rewind in local mode with the given source and +# target nodes and extra arguments. +# +# The target and source nodes are stopped before the call and the target is +# restarted afterward. The target's postgresql.conf is copied to a temporary +# location and passed to pg_rewind with --config-file, so that pg_rewind can +# update the target's config file in place without worrying about file +# permissions. The temporary config file is moved back to the target's data +# directory and permissions fixed after pg_rewind finishes. +sub rewind_node +{ + my ($target, $source, $label, @extra_args) = @_; + $source->stop; + $target->stop; + + my $tpgdata = $target->data_dir; + my $tmp = PostgreSQL::Test::Utils::tempdir; + copy("$tpgdata/postgresql.conf", "$tmp/target-postgresql.conf.tmp"); + + command_ok( + [ + 'pg_rewind', + '--debug', + '--source-pgdata' => $source->data_dir, + '--target-pgdata' => $target->data_dir, + '--no-sync', + '--config-file' => "$tmp/target-postgresql.conf.tmp", + @extra_args, + ], + $label); + + move("$tmp/target-postgresql.conf.tmp", "$tpgdata/postgresql.conf"); + chmod($target->group_access() ? 0640 : 0600, "$tpgdata/postgresql.conf") + or BAIL_OUT("unable to set permissions for $tpgdata/postgresql.conf"); + + $target->start; +} + +# Rewrite a node's TLI history file in the old 3-field format (no UUID), so +# that pg_rewind sees a zero UUID for that side, as if the node had been +# promoted by a server that predates the UUID feature. +sub strip_tli_uuid +{ + my ($node, $tli) = @_; + my $histfile = sprintf("%s/pg_wal/%08X.history", $node->data_dir, $tli); + open(my $fh, '<', $histfile) or die "cannot open $histfile: $!"; + my @lines = <$fh>; + close $fh; + open($fh, '>', $histfile) or die "cannot write $histfile: $!"; + for my $line (@lines) + { + chomp $line; + my @f = split(/\t/, $line, 4); + if (@f == 4) + { + # Drop the UUID field (index 2); keep parentTLI, switchpoint, reason. + print $fh join("\t", $f[0], $f[1], $f[3]) . "\n"; + } + else + { + print $fh "$line\n"; + } + } + close $fh; +} + +# Helper function to create an origin node with a test table and a row containing +# the given label. The node is started and ready for use as a source for +# standbys. +sub setup_origin +{ + my ($label) = @_; + my $node = PostgreSQL::Test::Cluster->new($label); + $node->init(allows_streaming => 1); + $node->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $node->start; + $node->safe_psql('postgres', "CREATE TABLE tbl (val text)"); + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); + return $node; +} + +# Helper function to create multiple standby nodes from the same origin node. +# Each standby gets its own backup and data directory, so that they will +# generate independent UUIDs on promotion even though they share the same +# timeline history up to the point of promotion. +sub setup_standbys_from_origin +{ + my ($origin, @names) = @_; + my @standbys; + for my $name (@names) + { + my $standby = PostgreSQL::Test::Cluster->new($name); + $origin->backup($standby->name); + $standby->init_from_backup($origin, $standby->name, + has_streaming => 1); + $standby->append_conf('postgresql.conf', "wal_keep_size = 320MB\n"); + $standby->set_standby_mode(); + $standby->start; + push @standbys, $standby; + } + return @standbys; +} + +# Helper function to wait for multiple standby nodes to catch up to the origin. +sub sync_standbys_with_origin +{ + my ($origin, @standbys) = @_; + $origin->wait_for_catchup($_) for @standbys; +} + +# Helper function to insert a row with the given label into a node's test table. +sub write_record +{ + my ($node, $label) = @_; + $node->safe_psql('postgres', "INSERT INTO tbl VALUES ('$label')"); + $node->safe_psql('postgres', 'CHECKPOINT'); +} + +# Test that pg_rewind detects and handles two standbys that independently +# promoted to the same timeline ID. Before the UUID-based divergence check, +# pg_rewind's same-TLI shortcut would incorrectly skip the rewind in this +# case, leaving the target's diverged WAL intact. +# +# origin (TLI 1) +# | +# +--- node_a (TLI 1) --promote--> TLI 2, UUID-A (target) +# | +# +--- node_b (TLI 1) --promote--> TLI 2, UUID-B (source) +# +# pg_rewind must detect the UUID mismatch and rewind node_a to match node_b. + +my $node_origin = setup_origin('origin'); + +# Create node_a and node_b from separate backups of origin so that each +# has its own data directory and will generate an independent UUID on promotion. +my ($node_a, $node_b) = + setup_standbys_from_origin($node_origin, 'node_a', 'node_b'); + +# Wait for both standbys to catch up to origin, then stop origin. After +# this point the two standbys are isolated and will promote independently. +sync_standbys_with_origin($node_origin, $node_a, $node_b); +$node_origin->stop; + +# Promote both standbys. Each lands on TLI 2 but generates a distinct UUID, +# so the resulting clusters are diverged even though they share a timeline ID. +$node_a->promote; +$node_b->promote; + +# Insert a divergent row on each so the rewind has visible work to do. +write_record($node_a, 'in A'); +write_record($node_b, 'in B'); + +rewind_node($node_a, $node_b, + 'pg_rewind detects independent same-TLI promotions'); + +my $result = + $node_a->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result, "in B\norigin", + 'rewound node has source data, not its own divergent data'); + +$node_a->teardown_node; +$node_b->teardown_node; +$node_origin->teardown_node; + +# Test that pg_rewind correctly rewinds across a TLI mismatch buried in a shared +# prefix of the timeline history. The target has gone through three timelines +# (TLI 1 -> TLI 2 -> TLI 3) while the source independently promoted from TLI 1 +# to what is numerically TLI 2 but with a different UUID (TLI 2'). The deepest +# common ancestor is therefore TLI 1, and pg_rewind must rewind the target all +# the way back to the end of TLI 1. +# +# origin (TLI 1) --+-- node_x --promote--> TLI 2 -- node_a --promote--> TLI 3 +# | (target: TLI 1->TLI 2->TLI 3) +# +-- node_b --promote--> TLI 2' +# (source: TLI 1->TLI 2') +# +# findCommonAncestorTimeline walks forward: TLI 1 entries match (UUID=0 on +# both sides), then TLI 2 vs TLI 2' match on tli and begin but differ on +# UUID, signalling independent promotions. The algorithm therefore backs up +# to TLI 1 as the common ancestor and sets the divergence point to the end +# of TLI 1. + +my $node_origin2 = setup_origin('origin2'); + +# node_x and node_b2 both start from the same TLI 1 baseline. +my ($node_x, $node_b2) = + setup_standbys_from_origin($node_origin2, 'node_x', 'node_b2'); + +# Both standbys must be caught up to the same LSN before origin stops, so +# that TLI 2 and TLI 2' both begin at the same WAL position. +sync_standbys_with_origin($node_origin2, $node_x, $node_b2); +$node_origin2->stop; + +# Promote node_x to TLI 2 (UUID-X) and insert a row. node_b2 is still on +# TLI 1 and has not yet seen any TLI 2 WAL. +$node_x->promote; +write_record($node_x, 'x'); + +# Build node_a2 as a standby of node_x, then promote it to TLI 3. +my ($node_a2) = setup_standbys_from_origin($node_x, 'node_a2'); + +sync_standbys_with_origin($node_x, $node_a2); +$node_x->stop; + +$node_a2->promote; + +# Now promote node_b2 independently from TLI 1 to TLI 2' (UUID-B, != UUID-X). +$node_b2->promote; +write_record($node_b2, 'b'); + +# Rewind node_a2 (TLI 1->TLI 2->TLI 3) from node_b2 (TLI 1->TLI 2') in +# local mode. The rewind must reach back to the end of TLI 1. +# +# node_a2 was initialised from a streaming backup of node_x taken after +# node_x had already completed segment 4 of TLI 2; that segment therefore +# does not appear in node_a2's pg_wal. pg_rewind's backward scan for the +# last checkpoint before the divergence point needs that segment, so we +# point restore_command at node_x's pg_wal and use --restore-target-wal. +# +# Note: no row is inserted on TLI 3. This is intentional: the only +# post-divergence table modification in the target's WAL is the 'x' INSERT +# on TLI 2. On unpatched code the WAL scan would start from the TLI 2 +# shutdown checkpoint (just before TLI 3), miss that earlier insert, and +# leave 'x' in place instead of replacing it with 'b'. +my $node_x_waldir = $node_x->data_dir . "/pg_wal"; +if ($PostgreSQL::Test::Utils::windows_os) +{ + $node_x_waldir =~ s{\\}{\\\\}g; + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'copy "$node_x_waldir\\\\%f" "%p"'\n)); +} +else +{ + $node_a2->append_conf('postgresql.conf', + qq(\nrestore_command = 'cp "$node_x_waldir/%f" "%p"'\n)); +} + +rewind_node($node_a2, $node_b2, + 'pg_rewind rewinds across mismatched TLI 2 / TLI 2-prime to TLI 1', + '--restore-target-wal'); +my $result2 = + $node_a2->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result2, "b\norigin2", + 'rewound node reflects source history, not target TLI 2/TLI 3 data'); + +$node_a2->teardown_node; +$node_b2->teardown_node; +$node_x->teardown_node; +$node_origin2->teardown_node; + +# Test that pg_rewind correctly detects a mismatch when one cluster's TLI 2 +# history entry carries a zero UUID (old-format history file) while the other +# carries a real UUID. The two clusters must have promoted independently, so +# pg_rewind must rewind to TLI 1 rather than accepting the same-TLI shortcut. +# +# Run both orientations: +# (a) target has zero UUID, source has real UUID +# (b) target has real UUID, source has zero UUID +# +# In both cases the setup is: +# +# origin (TLI 1) --+-- node_p --promote--> TLI 2, UUID-P (target) +# | +# +-- node_q --promote--> TLI 2, UUID-Q (source) +# +# One side then has its history file rewritten to the old 3-field format so +# that its UUID reads as zero. pg_rewind must treat zero-vs-nonzero as +# incompatible (they cannot be the same promotion) and rewind to TLI 1. + +for my $strip_target (1, 0) +{ + my $zero_side = $strip_target ? 'target' : 'source'; + my $real_side = $strip_target ? 'source' : 'target'; + my $sfx = $strip_target ? 'zt' : 'zs'; + my $label = + "pg_rewind rewinds when $zero_side has zero UUID and $real_side has real UUID"; + + my $node_origin3 = setup_origin("origin3_$sfx"); + my ($node_p, $node_q) = + setup_standbys_from_origin($node_origin3, "node_p_$sfx", "node_q_$sfx"); + + sync_standbys_with_origin($node_origin3, $node_p, $node_q); + $node_origin3->stop; + + $node_p->promote; + $node_q->promote; + + write_record($node_p, 'in P'); + write_record($node_q, 'in Q'); + + # Strip UUID from the chosen side to simulate a pre-UUID server. + strip_tli_uuid($strip_target ? $node_p : $node_q, 2); + + rewind_node($node_p, $node_q, $label); + my $result3 = + $node_p->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); + is( $result3, + "in Q\norigin3_$sfx", + 'rewound node has source data, not its own divergent row'); + + $node_p->teardown_node; + $node_q->teardown_node; + $node_origin3->teardown_node; +} + +# Test that pg_rewind detects independent promotions to TLI 3 when both +# clusters share a common TLI 1 -> TLI 2 history (same UUID) but independently +# promoted from TLI 2 to TLI 3, producing different TLI 3 UUIDs. +# +# origin (TLI 1) --- node_mid --promote--> TLI 2, UUID-M +# | +# +-- node_c --promote--> TLI 3, UUID-C (target) +# | +# +-- node_d --promote--> TLI 3', UUID-D (source) +# +# The same-TLI shortcut compares entry[Nentries-2].tluuid on each side; that +# is the UUID of the TLI 3 promotion, which differs. The full rewind path +# then walks the history forward: TLI 1 matches (same tli/begin/UUID-M at +# entry[0]), TLI 2 also matches (same tli/begin; UUID-M is the same on both +# sides at entry[0]), but TLI 3 vs TLI 3' differ at entry[1] (UUID-C != UUID-D), +# so the divergence point is set to the end of TLI 2. + +my $node_origin4 = setup_origin('origin4'); +my ($node_mid) = setup_standbys_from_origin($node_origin4, 'node_mid'); + +sync_standbys_with_origin($node_origin4, $node_mid); +$node_origin4->stop; + +# Promote node_mid to TLI 2 and insert a row that both TLI 3 nodes will share. +$node_mid->promote; +write_record($node_mid, 'mid'); + +# node_c and node_d both start as standbys of node_mid so they share the same +# TLI 2 promotion UUID (UUID-M). +my ($node_c, $node_d) = + setup_standbys_from_origin($node_mid, 'node_c', 'node_d'); +sync_standbys_with_origin($node_mid, $node_c, $node_d); +$node_mid->stop; + +# Promote both independently; each generates a distinct TLI 3 UUID. +$node_c->promote; +$node_d->promote; + +write_record($node_c, 'c'); +write_record($node_d, 'd'); + +rewind_node($node_c, $node_d, + 'pg_rewind detects independent TLI 3 / TLI 3-prime promotions sharing TLI 2' +); +my $result4 = + $node_c->safe_psql('postgres', "SELECT val FROM tbl ORDER BY val"); +is($result4, "d\nmid\norigin4", + 'rewound node has source TLI 3-prime data, not its own TLI 3 data'); + +$node_c->teardown_node; +$node_d->teardown_node; +$node_mid->teardown_node; +$node_origin4->teardown_node; + done_testing(); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index dda06eaa0bc..b6500606b27 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -9,9 +9,40 @@ */ #include "postgres_fe.h" +#include <ctype.h> +#include <string.h> + #include "access/timeline.h" #include "pg_rewind.h" +/* + * Parse a UUID string in standard dashed form into a pg_uuid_t. + * Returns true on success, false if str is not a valid UUID string. + */ +static bool +rewind_parse_uuid(const char *str, pg_uuid_t *uuid) +{ + const char *src = str; + + for (int i = 0; i < UUID_LEN; i++) + { + char buf[3]; + + if (!isxdigit((unsigned char) src[0]) || + !isxdigit((unsigned char) src[1])) + return false; + buf[0] = src[0]; + buf[1] = src[1]; + buf[2] = '\0'; + uuid->data[i] = (unsigned char) strtoul(buf, NULL, 16); + src += 2; + /* skip dash at positions after bytes 3, 5, 7, 9 (i == 3,5,7,9) */ + if (src[0] == '-' && (i == 3 || i == 5 || i == 7 || i == 9)) + src++; + } + return (*src == '\0'); +} + /* * This is copy-pasted from the backend readTimeLineHistory, modified to * return a malloc'd array and to work without backend functions. @@ -48,6 +79,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; + char uuid_str[UUID_STR_LEN + 1] = {0}; fline = bufptr; while (*bufptr && *bufptr != '\n') @@ -66,7 +98,8 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(fline, "%u\t%X/%08X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(fline, "%u\t%X/%08X\t%36s", &tli, &switchpoint_hi, + &switchpoint_lo, uuid_str); if (nfields < 1) { @@ -75,7 +108,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) pg_log_error_detail("Expected a numeric timeline ID."); exit(1); } - if (nfields != 3) + if (nfields < 3) { pg_log_error("syntax error in history file: %s", fline); pg_log_error_detail("Expected a write-ahead log switchpoint location."); @@ -99,7 +132,14 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->end = ((uint64) (switchpoint_hi)) << 32 | (uint64) switchpoint_lo; prevend = entry->end; - /* we ignore the remainder of each line */ + /* + * Parse the optional UUID field. Old history files have the reason + * string in field 4; its first word is much shorter than UUID_STR_LEN + * so the length check safely distinguishes old from new format. + */ + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); + if (nfields == 4 && strlen(uuid_str) == UUID_STR_LEN) + rewind_parse_uuid(uuid_str, &entry->tluuid); } if (entries && targetTLI <= lasttli) @@ -123,6 +163,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) entry->tli = targetTLI; entry->begin = prevend; entry->end = InvalidXLogRecPtr; + memset(&entry->tluuid, 0, sizeof(pg_uuid_t)); *nentries = nlines; return entries; diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 3aee3419a5c..3b94c4f0a4d 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -13,6 +13,7 @@ #include "access/xlogdefs.h" #include "nodes/pg_list.h" +#include "utils/uuid.h" /* * A list of these structs describes the timeline history of the server. Each @@ -22,9 +23,10 @@ * pointers of all the entries form a contiguous line from beginning of time * to infinity. */ -typedef struct +typedef struct TimeLineHistoryEntry { TimeLineID tli; + pg_uuid_t tluuid; /* from history file; zero if unknown */ XLogRecPtr begin; /* inclusive */ XLogRecPtr end; /* exclusive, InvalidXLogRecPtr means infinity */ } TimeLineHistoryEntry; @@ -33,6 +35,7 @@ extern List *readTimeLineHistory(TimeLineID targetTLI); extern bool existsTimeLineHistory(TimeLineID probeTLI); extern TimeLineID findNewestTimeLine(TimeLineID startTLI); extern void writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, + const pg_uuid_t *newTLUUID, XLogRecPtr switchpoint, const char *reason); extern void writeTimeLineHistoryFile(TimeLineID tli, const char *content, size_t size); extern void restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401..588094090c8 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -22,6 +22,7 @@ #include "access/xlogdefs.h" #include "access/xlogreader.h" #include "datatype/timestamp.h" +#include "utils/uuid.h" #include "lib/stringinfo.h" #include "pgtime.h" #include "storage/block.h" diff --git a/src/include/utils/uuid.h b/src/include/utils/uuid.h index 572d8cf4c36..6839de2e0b2 100644 --- a/src/include/utils/uuid.h +++ b/src/include/utils/uuid.h @@ -17,12 +17,16 @@ /* uuid size in bytes */ #define UUID_LEN 16 +/* length of a UUID string (without null terminator): xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */ +#define UUID_STR_LEN 36 + typedef struct pg_uuid_t { unsigned char data[UUID_LEN]; } pg_uuid_t; -/* fmgr interface macros */ +/* fmgr interface macros (backend only) */ +#ifndef FRONTEND static inline Datum UUIDPGetDatum(const pg_uuid_t *X) { @@ -38,5 +42,9 @@ DatumGetUUIDP(Datum X) } #define PG_GETARG_UUID_P(X) DatumGetUUIDP(PG_GETARG_DATUM(X)) +#endif /* !FRONTEND */ + +extern pg_uuid_t *generate_uuidv7(uint64 unix_ts_ms, uint32 sub_ms); +extern pg_uuid_t *generate_uuidv7_r(pg_uuid_t *uuid, uint64 unix_ts_ms, uint32 sub_ms); #endif /* UUID_H */ -- 2.53.0 --=-=-=-- ^ permalink raw reply [nested|flat] 330+ messages in thread
* [PATCH 4/4] Use singular "datachecksum" consistently in process names @ 2026-05-26 07:56 Kyotaro Horiguchi <horikyota.ntt@gmail.com> 0 siblings, 0 replies; 330+ messages in thread From: Kyotaro Horiguchi @ 2026-05-26 07:56 UTC (permalink / raw) datachecksum_state.c currently refers to processes as both "datachecksum worker/launcher" and "datachecksums launcher". Use the singular "datachecksum" form consistently for process names instead. --- src/backend/access/transam/xlog.c | 4 +- src/backend/postmaster/bgworker.c | 8 +- src/backend/postmaster/datachecksum_state.c | 234 ++++++++++---------- src/backend/postmaster/postmaster.c | 4 +- src/backend/storage/ipc/procsignal.c | 2 +- src/backend/utils/activity/pgstat_backend.c | 4 +- src/backend/utils/activity/pgstat_io.c | 4 +- src/backend/utils/init/miscinit.c | 2 +- src/backend/utils/init/postinit.c | 2 +- src/include/miscadmin.h | 10 +- src/include/postmaster/datachecksum_state.h | 30 +-- src/include/postmaster/proctypelist.h | 4 +- src/include/storage/subsystemlist.h | 2 +- 13 files changed, 155 insertions(+), 155 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index fecdf0d4b05..16f8278809e 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -9178,7 +9178,7 @@ xlog_redo(XLogReaderState *record) SpinLockRelease(&XLogCtl->info_lck); if (new_state) - EmitAndWaitDataChecksumsBarrier(redo_rec.data_checksum_version); + EmitAndWaitDataChecksumBarrier(redo_rec.data_checksum_version); } else if (info == XLOG_LOGICAL_DECODING_STATUS_CHANGE) { @@ -9256,7 +9256,7 @@ xlog2_redo(XLogReaderState *record) * change to checksum status. Once the barrier has been passed we can * initiate the corresponding processing. */ - EmitAndWaitDataChecksumsBarrier(state.new_checksum_state); + EmitAndWaitDataChecksumBarrier(state.new_checksum_state); } } diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 2e4acad4f00..68256d90cef 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -160,12 +160,12 @@ static const struct .fn_addr = TableSyncWorkerMain }, { - .fn_name = "DataChecksumsWorkerLauncherMain", - .fn_addr = DataChecksumsWorkerLauncherMain + .fn_name = "DataChecksumWorkerLauncherMain", + .fn_addr = DataChecksumWorkerLauncherMain }, { - .fn_name = "DataChecksumsWorkerMain", - .fn_addr = DataChecksumsWorkerMain + .fn_name = "DataChecksumWorkerMain", + .fn_addr = DataChecksumWorkerMain } }; diff --git a/src/backend/postmaster/datachecksum_state.c b/src/backend/postmaster/datachecksum_state.c index 33430147ff2..1715e506cbc 100644 --- a/src/backend/postmaster/datachecksum_state.c +++ b/src/backend/postmaster/datachecksum_state.c @@ -26,7 +26,7 @@ * checksums enabled, then disabled them and updated the page while they were * disabled. * - * The DataChecksumsWorker will compile a list of all databases at the start, + * The DataChecksumWorker will compile a list of all databases at the start, * any databases created concurrently will see the in-progress state and will * be checksummed automatically. All databases from the original list MUST BE * successfully processed in order for data checksums to be enabled, the only @@ -99,10 +99,10 @@ * state will also be set to "off". * * Backends transition Bd -> Bi via a procsignalbarrier which is emitted by the - * DataChecksumsWorkerLauncherMain. When all backends have acknowledged the + * DataChecksumWorkerLauncherMain. When all backends have acknowledged the * barrier then Bd will be empty and the next phase can begin: calculating and - * writing data checksums with DataChecksumsWorkers. When the - * DataChecksumsWorker processes have finished writing checksums on all pages, + * writing data checksums with DataChecksumWorkers. When the + * DataChecksumWorker processes have finished writing checksums on all pages, * data checksums are enabled cluster-wide via another procsignalbarrier. * There are four sets of backends where Bd shall be an empty set: * @@ -155,7 +155,7 @@ * found on the -hackers threads linked to in the commit message of this * feature. * - * * Launching datachecksumsworker for resuming operation from the startup + * * Launching datachecksumworker for resuming operation from the startup * process: Currently users have to restart processing manually after a * restart since dynamic background worker cannot be started from the * postmaster. Changing the startup process could make restarting the @@ -280,15 +280,15 @@ static const ChecksumBarrierCondition checksum_barriers[9] = * Signaling between backends calling pg_enable/disable_data_checksums, the * checksums launcher process, and the checksums worker process. * - * This struct is protected by DataChecksumsWorkerLock + * This struct is protected by DataChecksumWorkerLock */ -typedef struct DataChecksumsStateStruct +typedef struct DataChecksumStateStruct { /* * These are set by pg_{enable|disable}_data_checksums, to tell the * launcher what the target state is. */ - DataChecksumsWorkerOperation launch_operation; + DataChecksumWorkerOperation launch_operation; int launch_cost_delay; int launch_cost_limit; @@ -315,7 +315,7 @@ typedef struct DataChecksumsStateStruct * without a lock. If multiple workers, or dynamic cost parameters, are * supported at some point then this would need to be revisited. */ - DataChecksumsWorkerOperation operation; + DataChecksumWorkerOperation operation; int cost_delay; int cost_limit; @@ -329,58 +329,58 @@ typedef struct DataChecksumsStateStruct */ /* result, set by worker before exiting */ - DataChecksumsWorkerResult success; + DataChecksumWorkerResult success; /* * Tells the worker process whether it should also process the shared * catalogs */ bool process_shared_catalogs; -} DataChecksumsStateStruct; +} DataChecksumStateStruct; -/* Shared memory segment for datachecksumsworker */ -static DataChecksumsStateStruct *DataChecksumState; +/* Shared memory segment for datachecksumworker */ +static DataChecksumStateStruct *DataChecksumState; -typedef struct DataChecksumsWorkerDatabase +typedef struct DataChecksumWorkerDatabase { Oid dboid; char *dbname; -} DataChecksumsWorkerDatabase; +} DataChecksumWorkerDatabase; /* Flag set by the interrupt handler */ static volatile sig_atomic_t abort_requested = false; /* - * Have we set the DataChecksumsStateStruct->launcher_running flag? + * Have we set the DataChecksumStateStruct->launcher_running flag? * If we have, we need to clear it before exiting! */ static volatile sig_atomic_t launcher_running = false; /* Are we enabling data checksums, or disabling them? */ -static DataChecksumsWorkerOperation operation; +static DataChecksumWorkerOperation operation; /* Prototypes */ -static void DataChecksumsShmemRequest(void *arg); +static void DataChecksumShmemRequest(void *arg); static bool DatabaseExists(Oid dboid); static List *BuildDatabaseList(void); static List *BuildRelationList(bool temp_relations, bool include_shared); static void FreeDatabaseList(List *dblist); -static DataChecksumsWorkerResult ProcessDatabase(DataChecksumsWorkerDatabase *db); +static DataChecksumWorkerResult ProcessDatabase(DataChecksumWorkerDatabase *db); static bool ProcessAllDatabases(void); static bool ProcessSingleRelationFork(Relation reln, ForkNumber forkNum, BufferAccessStrategy strategy); static void launcher_cancel_handler(SIGNAL_ARGS); static void WaitForAllTransactionsToFinish(void); -const ShmemCallbacks DataChecksumsShmemCallbacks = { - .request_fn = DataChecksumsShmemRequest, +const ShmemCallbacks DataChecksumShmemCallbacks = { + .request_fn = DataChecksumShmemRequest, }; #define CHECK_FOR_ABORT_REQUEST() \ do { \ - LWLockAcquire(DataChecksumsWorkerLock, LW_SHARED); \ + LWLockAcquire(DataChecksumWorkerLock, LW_SHARED); \ if (DataChecksumState->launch_operation != operation) \ abort_requested = true; \ - LWLockRelease(DataChecksumsWorkerLock); \ + LWLockRelease(DataChecksumWorkerLock); \ } while (0) @@ -389,7 +389,7 @@ const ShmemCallbacks DataChecksumsShmemCallbacks = { */ void -EmitAndWaitDataChecksumsBarrier(uint32 state) +EmitAndWaitDataChecksumBarrier(uint32 state) { uint64 barrier; @@ -421,7 +421,7 @@ EmitAndWaitDataChecksumsBarrier(uint32 state) } /* - * AbsorbDataChecksumsBarrier + * AbsorbDataChecksumBarrier * Generic function for absorbing data checksum state changes * * All procsignalbarriers regarding data checksum state changes are absorbed @@ -430,7 +430,7 @@ EmitAndWaitDataChecksumsBarrier(uint32 state) * used to look up the relevant entry. */ bool -AbsorbDataChecksumsBarrier(ProcSignalBarrierType barrier) +AbsorbDataChecksumBarrier(ProcSignalBarrierType barrier) { uint32 target_state; int current = data_checksums; @@ -516,7 +516,7 @@ disable_data_checksums(PG_FUNCTION_ARGS) errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser to change data checksum state")); - StartDataChecksumsWorkerLauncher(DISABLE_DATACHECKSUMS, 0, 0); + StartDataChecksumWorkerLauncher(DISABLE_DATACHECKSUMS, 0, 0); PG_RETURN_VOID(); } @@ -548,27 +548,27 @@ enable_data_checksums(PG_FUNCTION_ARGS) errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cost limit must be greater than zero")); - StartDataChecksumsWorkerLauncher(ENABLE_DATACHECKSUMS, cost_delay, cost_limit); + StartDataChecksumWorkerLauncher(ENABLE_DATACHECKSUMS, cost_delay, cost_limit); PG_RETURN_VOID(); } /***************************************************************************** - * Functionality for running the datachecksumsworker and associated launcher + * Functionality for running the datachecksum worker and associated launcher */ /* - * StartDataChecksumsWorkerLauncher - * Main entry point for datachecksumsworker launcher process + * StartDataChecksumWorkerLauncher + * Main entry point for datachecksumworker launcher process * * The main entrypoint for starting data checksums processing for enabling as * well as disabling. */ void -StartDataChecksumsWorkerLauncher(DataChecksumsWorkerOperation op, - int cost_delay, - int cost_limit) +StartDataChecksumWorkerLauncher(DataChecksumWorkerOperation op, + int cost_delay, + int cost_limit) { BackgroundWorker bgw; BackgroundWorkerHandle *bgw_handle; @@ -580,10 +580,10 @@ StartDataChecksumsWorkerLauncher(DataChecksumsWorkerOperation op, Assert(cost_delay == 0 && cost_limit == 0); #endif - INJECTION_POINT("datachecksumsworker-startup-delay", NULL); + INJECTION_POINT("datachecksumworker-startup-delay", NULL); /* Store the desired state in shared memory */ - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); DataChecksumState->launch_operation = op; DataChecksumState->launch_cost_delay = cost_delay; @@ -592,7 +592,7 @@ StartDataChecksumsWorkerLauncher(DataChecksumsWorkerOperation op, /* Is the launcher already running? If so, what is it doing? */ running = DataChecksumState->launcher_running; - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); /* * Launch a new launcher process, if it's not running already. @@ -625,7 +625,7 @@ StartDataChecksumsWorkerLauncher(DataChecksumsWorkerOperation op, bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres"); - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "DataChecksumsWorkerLauncherMain"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "DataChecksumWorkerLauncherMain"); snprintf(bgw.bgw_name, BGW_MAXLEN, "datachecksum launcher"); snprintf(bgw.bgw_type, BGW_MAXLEN, "datachecksum launcher"); bgw.bgw_restart_time = BGW_NEVER_RESTART; @@ -715,10 +715,10 @@ ProcessSingleRelationFork(Relation reln, ForkNumber forkNum, BufferAccessStrateg * abortion will bubble up from here. */ Assert(operation == ENABLE_DATACHECKSUMS); - LWLockAcquire(DataChecksumsWorkerLock, LW_SHARED); + LWLockAcquire(DataChecksumWorkerLock, LW_SHARED); if (DataChecksumState->launch_operation == DISABLE_DATACHECKSUMS) abort_requested = true; - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); if (abort_requested) return false; @@ -795,8 +795,8 @@ ProcessSingleRelationByOid(Oid relationId, BufferAccessStrategy strategy) * waiting for it to finish. We have to do this in a separate worker, since * each process can only be connected to one database during its lifetime. */ -static DataChecksumsWorkerResult -ProcessDatabase(DataChecksumsWorkerDatabase *db) +static DataChecksumWorkerResult +ProcessDatabase(DataChecksumWorkerDatabase *db) { BackgroundWorker bgw; BackgroundWorkerHandle *bgw_handle; @@ -804,15 +804,15 @@ ProcessDatabase(DataChecksumsWorkerDatabase *db) pid_t pid; char activity[NAMEDATALEN + 64]; - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); - DataChecksumState->success = DATACHECKSUMSWORKER_FAILED; - LWLockRelease(DataChecksumsWorkerLock); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); + DataChecksumState->success = DATACHECKSUMWORKER_FAILED; + LWLockRelease(DataChecksumWorkerLock); memset(&bgw, 0, sizeof(bgw)); bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres"); - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "%s", "DataChecksumsWorkerMain"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "%s", "DataChecksumWorkerMain"); snprintf(bgw.bgw_name, BGW_MAXLEN, "datachecksum worker"); snprintf(bgw.bgw_type, BGW_MAXLEN, "datachecksum worker"); bgw.bgw_restart_time = BGW_NEVER_RESTART; @@ -830,7 +830,7 @@ ProcessDatabase(DataChecksumsWorkerDatabase *db) errmsg("could not start background worker for enabling data checksums in database \"%s\"", db->dbname), errhint("The \"%s\" setting might be too low.", "max_worker_processes")); - return DATACHECKSUMSWORKER_FAILED; + return DATACHECKSUMWORKER_FAILED; } status = WaitForBackgroundWorkerStartup(bgw_handle, &pid); @@ -840,17 +840,17 @@ ProcessDatabase(DataChecksumsWorkerDatabase *db) * If the worker managed to start, and stop, before we got to waiting * for it we can see a STOPPED status here without it being a failure. */ - LWLockAcquire(DataChecksumsWorkerLock, LW_SHARED); - if (DataChecksumState->success == DATACHECKSUMSWORKER_SUCCESSFUL) + LWLockAcquire(DataChecksumWorkerLock, LW_SHARED); + if (DataChecksumState->success == DATACHECKSUMWORKER_SUCCESSFUL) { - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); pgstat_report_activity(STATE_IDLE, NULL); - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); DataChecksumState->worker_pid = InvalidPid; - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); return DataChecksumState->success; } - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); ereport(WARNING, errmsg("could not start background worker for enabling data checksums in database \"%s\"", @@ -862,9 +862,9 @@ ProcessDatabase(DataChecksumsWorkerDatabase *db) * treat it as not an error, else treat as fatal and error out. */ if (DatabaseExists(db->dboid)) - return DATACHECKSUMSWORKER_FAILED; + return DATACHECKSUMWORKER_FAILED; else - return DATACHECKSUMSWORKER_DROPDB; + return DATACHECKSUMWORKER_DROPDB; } /* @@ -886,9 +886,9 @@ ProcessDatabase(DataChecksumsWorkerDatabase *db) db->dbname)); /* Save the pid of the worker so we can signal it later */ - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); DataChecksumState->worker_pid = pid; - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); snprintf(activity, sizeof(activity) - 1, "Waiting for worker in database %s (pid %ld)", db->dbname, (long) pid); @@ -902,17 +902,17 @@ ProcessDatabase(DataChecksumsWorkerDatabase *db) db->dbname), errhint("Restart the database and restart data checksum processing by calling pg_enable_data_checksums().")); - LWLockAcquire(DataChecksumsWorkerLock, LW_SHARED); - if (DataChecksumState->success == DATACHECKSUMSWORKER_ABORTED) + LWLockAcquire(DataChecksumWorkerLock, LW_SHARED); + if (DataChecksumState->success == DATACHECKSUMWORKER_ABORTED) ereport(LOG, errmsg("data checksums processing was aborted in database \"%s\"", db->dbname)); - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); pgstat_report_activity(STATE_IDLE, NULL); - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); DataChecksumState->worker_pid = InvalidPid; - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); return DataChecksumState->success; } @@ -934,14 +934,14 @@ launcher_exit(int code, Datum arg) if (launcher_running) { - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); if (DataChecksumState->worker_pid != InvalidPid) { ereport(LOG, errmsg("data checksums launcher exiting while worker is still running, signalling worker")); kill(DataChecksumState->worker_pid, SIGTERM); } - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); } /* @@ -951,10 +951,10 @@ launcher_exit(int code, Datum arg) if (DataChecksumsInProgressOn()) SetDataChecksumsOff(); - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); launcher_running = false; DataChecksumState->launcher_running = false; - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); } /* @@ -1044,7 +1044,7 @@ WaitForAllTransactionsToFinish(void) } /* - * DataChecksumsWorkerLauncherMain + * DataChecksumWorkerLauncherMain * * Main function for launching dynamic background workers for processing data * checksums in databases. This function has the bgworker management, with @@ -1052,11 +1052,11 @@ WaitForAllTransactionsToFinish(void) * initiating processing. */ void -DataChecksumsWorkerLauncherMain(Datum arg) +DataChecksumWorkerLauncherMain(Datum arg) { ereport(DEBUG1, - errmsg("background worker \"datachecksums launcher\" started")); + errmsg("background worker \"datachecksum launcher\" started")); pqsignal(SIGTERM, die); pqsignal(SIGINT, launcher_cancel_handler); @@ -1065,19 +1065,19 @@ DataChecksumsWorkerLauncherMain(Datum arg) BackgroundWorkerUnblockSignals(); - MyBackendType = B_DATACHECKSUMSWORKER_LAUNCHER; + MyBackendType = B_DATACHECKSUMWORKER_LAUNCHER; init_ps_display(NULL); - INJECTION_POINT("datachecksumsworker-launcher-delay", NULL); + INJECTION_POINT("datachecksumworker-launcher-delay", NULL); - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); if (DataChecksumState->launcher_running) { ereport(LOG, - errmsg("background worker \"datachecksums launcher\" already running, exiting")); + errmsg("background worker \"datachecksum launcher\" already running, exiting")); /* Launcher was already running, let it finish */ - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); return; } @@ -1092,7 +1092,7 @@ DataChecksumsWorkerLauncherMain(Datum arg) DataChecksumState->operation = operation; DataChecksumState->cost_delay = DataChecksumState->launch_cost_delay; DataChecksumState->cost_limit = DataChecksumState->launch_cost_limit; - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); /* * The target state can change while we are busy enabling/disabling @@ -1134,13 +1134,13 @@ again: * If the target state changed during processing then it's not a * failure, so restart processing instead. */ - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); if (DataChecksumState->launch_operation != operation) { - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); goto done; } - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); ereport(ERROR, errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("unable to enable data checksums in cluster")); @@ -1183,14 +1183,14 @@ done: * while we were running. In that case we will have to start all over * again. */ - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); if (DataChecksumState->launch_operation != operation) { DataChecksumState->operation = DataChecksumState->launch_operation; operation = DataChecksumState->launch_operation; DataChecksumState->cost_delay = DataChecksumState->launch_cost_delay; DataChecksumState->cost_limit = DataChecksumState->launch_cost_limit; - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); goto again; } @@ -1199,7 +1199,7 @@ done: launcher_running = false; DataChecksumState->launcher_running = false; - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); } /* @@ -1217,9 +1217,9 @@ ProcessAllDatabases(void) int cumulative_total = 0; /* Set up so first run processes shared catalogs, not once in every db */ - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); DataChecksumState->process_shared_catalogs = true; - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); /* Get a list of all databases to process */ WaitForAllTransactionsToFinish(); @@ -1254,18 +1254,18 @@ ProcessAllDatabases(void) pgstat_progress_update_multi_param(6, index, vals); } - foreach_ptr(DataChecksumsWorkerDatabase, db, DatabaseList) + foreach_ptr(DataChecksumWorkerDatabase, db, DatabaseList) { - DataChecksumsWorkerResult result; + DataChecksumWorkerResult result; result = ProcessDatabase(db); #ifdef USE_INJECTION_POINTS /* Allow a test process to alter the result of the operation */ - if (IS_INJECTION_POINT_ATTACHED("datachecksumsworker-fail-db-result")) + if (IS_INJECTION_POINT_ATTACHED("datachecksumworker-fail-db-result")) { - result = DATACHECKSUMSWORKER_FAILED; - INJECTION_POINT_CACHED("datachecksumsworker-fail-db-result", + result = DATACHECKSUMWORKER_FAILED; + INJECTION_POINT_CACHED("datachecksumworker-fail-db-result", db->dbname); } #endif @@ -1273,7 +1273,7 @@ ProcessAllDatabases(void) pgstat_progress_update_param(PROGRESS_DATACHECKSUMS_DBS_DONE, ++cumulative_total); - if (result == DATACHECKSUMSWORKER_FAILED) + if (result == DATACHECKSUMWORKER_FAILED) { /* * Disable checksums on cluster, because we failed one of the @@ -1285,7 +1285,7 @@ ProcessAllDatabases(void) errmsg("data checksums failed to get enabled in all databases, aborting"), errhint("The server log might have more information on the cause of the error.")); } - else if (result == DATACHECKSUMSWORKER_ABORTED || abort_requested) + else if (result == DATACHECKSUMWORKER_ABORTED || abort_requested) { /* Abort flag set, so exit the whole process */ return false; @@ -1295,9 +1295,9 @@ ProcessAllDatabases(void) * When one database has completed, it will have done shared catalogs * so we don't have to process them again. */ - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); DataChecksumState->process_shared_catalogs = false; - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); } FreeDatabaseList(DatabaseList); @@ -1308,14 +1308,14 @@ ProcessAllDatabases(void) } /* - * DataChecksumsShmemRequest - * Request datachecksumsworker-related shared memory + * DataChecksumShmemRequest + * Request datachecksumworker-related shared memory */ static void -DataChecksumsShmemRequest(void *arg) +DataChecksumShmemRequest(void *arg) { - ShmemRequestStruct(.name = "DataChecksumsWorker Data", - .size = sizeof(DataChecksumsStateStruct), + ShmemRequestStruct(.name = "DataChecksumWorker Data", + .size = sizeof(DataChecksumStateStruct), .ptr = (void **) &DataChecksumState, ); } @@ -1370,7 +1370,7 @@ DatabaseExists(Oid dboid) * BuildDatabaseList * Compile a list of all currently available databases in the cluster * - * This creates the list of databases for the datachecksumsworker workers to + * This creates the list of databases for the datachecksum workers to * add checksums to. If the caller wants to ensure that no concurrently * running CREATE DATABASE calls exist, this needs to be preceded by a call * to WaitForAllTransactionsToFinish(). @@ -1393,11 +1393,11 @@ BuildDatabaseList(void) while (HeapTupleIsValid(tup = heap_getnext(scan, ForwardScanDirection))) { Form_pg_database pgdb = (Form_pg_database) GETSTRUCT(tup); - DataChecksumsWorkerDatabase *db; + DataChecksumWorkerDatabase *db; oldctx = MemoryContextSwitchTo(ctx); - db = (DataChecksumsWorkerDatabase *) palloc0(sizeof(DataChecksumsWorkerDatabase)); + db = (DataChecksumWorkerDatabase *) palloc0(sizeof(DataChecksumWorkerDatabase)); db->dboid = pgdb->oid; db->dbname = pstrdup(NameStr(pgdb->datname)); @@ -1421,7 +1421,7 @@ FreeDatabaseList(List *dblist) if (!dblist) return; - foreach_ptr(DataChecksumsWorkerDatabase, db, dblist) + foreach_ptr(DataChecksumWorkerDatabase, db, dblist) { if (db->dbname != NULL) pfree(db->dbname); @@ -1496,7 +1496,7 @@ BuildRelationList(bool temp_relations, bool include_shared) } /* - * DataChecksumsWorkerMain + * DataChecksumWorkerMain * * Main function for enabling checksums in a single database. This is the * function set as the bgw_function_name in the dynamic background worker @@ -1507,7 +1507,7 @@ BuildRelationList(bool temp_relations, bool include_shared) * existing temporary relations with data checksums. */ void -DataChecksumsWorkerMain(Datum arg) +DataChecksumWorkerMain(Datum arg) { Oid dboid = DatumGetObjectId(arg); List *RelationList = NIL; @@ -1526,7 +1526,7 @@ DataChecksumsWorkerMain(Datum arg) BackgroundWorkerUnblockSignals(); - MyBackendType = B_DATACHECKSUMSWORKER_WORKER; + MyBackendType = B_DATACHECKSUMWORKER_WORKER; init_ps_display(NULL); BackgroundWorkerInitializeConnectionByOid(dboid, InvalidOid, @@ -1606,7 +1606,7 @@ DataChecksumsWorkerMain(Datum arg) * to reflect the new values and signal that the access strategy needs * to be refreshed. */ - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); if ((DataChecksumState->launch_cost_delay != DataChecksumState->cost_delay) || (DataChecksumState->launch_cost_limit != DataChecksumState->cost_limit)) { @@ -1620,7 +1620,7 @@ DataChecksumsWorkerMain(Datum arg) } else costs_updated = false; - LWLockRelease(DataChecksumsWorkerLock); + LWLockRelease(DataChecksumWorkerLock); if (costs_updated) { @@ -1634,9 +1634,9 @@ DataChecksumsWorkerMain(Datum arg) if (aborted || abort_requested) { - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); - DataChecksumState->success = DATACHECKSUMSWORKER_ABORTED; - LWLockRelease(DataChecksumsWorkerLock); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); + DataChecksumState->success = DATACHECKSUMWORKER_ABORTED; + LWLockRelease(DataChecksumWorkerLock); ereport(DEBUG1, errmsg("data checksum processing aborted in database OID %u", dboid)); @@ -1669,7 +1669,7 @@ DataChecksumsWorkerMain(Datum arg) list_free(CurrentTempTables); #ifdef USE_INJECTION_POINTS - if (IS_INJECTION_POINT_ATTACHED("datachecksumsworker-fake-temptable-wait")) + if (IS_INJECTION_POINT_ATTACHED("datachecksumworker-fake-temptable-wait")) { /* Make sure to just cause one retry */ if (!retried && numleft == 0) @@ -1677,7 +1677,7 @@ DataChecksumsWorkerMain(Datum arg) numleft = 1; retried = true; - INJECTION_POINT_CACHED("datachecksumsworker-fake-temptable-wait", NULL); + INJECTION_POINT_CACHED("datachecksumworker-fake-temptable-wait", NULL); } } #endif @@ -1706,9 +1706,9 @@ DataChecksumsWorkerMain(Datum arg) if (aborted || abort_requested) { - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); - DataChecksumState->success = DATACHECKSUMSWORKER_ABORTED; - LWLockRelease(DataChecksumsWorkerLock); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); + DataChecksumState->success = DATACHECKSUMWORKER_ABORTED; + LWLockRelease(DataChecksumWorkerLock); ereport(LOG, errmsg("data checksum processing aborted in database OID %u", dboid)); @@ -1721,7 +1721,7 @@ DataChecksumsWorkerMain(Datum arg) /* worker done */ pgstat_progress_end_command(); - LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); - DataChecksumState->success = DATACHECKSUMSWORKER_SUCCESSFUL; - LWLockRelease(DataChecksumsWorkerLock); + LWLockAcquire(DataChecksumWorkerLock, LW_EXCLUSIVE); + DataChecksumState->success = DATACHECKSUMWORKER_SUCCESSFUL; + LWLockRelease(DataChecksumWorkerLock); } diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 90c7c4528e8..0debcb18991 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -3010,8 +3010,8 @@ PostmasterStateMachine(void) /* also add data checksums processes */ remainMask = btmask_add(remainMask, - B_DATACHECKSUMSWORKER_LAUNCHER, - B_DATACHECKSUMSWORKER_WORKER); + B_DATACHECKSUMWORKER_LAUNCHER, + B_DATACHECKSUMWORKER_WORKER); /* All types should be included in targetMask or remainMask */ Assert((remainMask.mask | targetMask.mask) == BTYPE_MASK_ALL.mask); diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index 1397f65f67b..0e0c7acf0df 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -596,7 +596,7 @@ ProcessProcSignalBarrier(void) case PROCSIGNAL_BARRIER_CHECKSUM_ON: case PROCSIGNAL_BARRIER_CHECKSUM_INPROGRESS_OFF: case PROCSIGNAL_BARRIER_CHECKSUM_OFF: - processed = AbsorbDataChecksumsBarrier(type); + processed = AbsorbDataChecksumBarrier(type); break; } diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c index 73461c9bca5..bccf6f27121 100644 --- a/src/backend/utils/activity/pgstat_backend.c +++ b/src/backend/utils/activity/pgstat_backend.c @@ -381,8 +381,8 @@ pgstat_tracks_backend_bktype(BackendType bktype) case B_CHECKPOINTER: case B_IO_WORKER: case B_STARTUP: - case B_DATACHECKSUMSWORKER_LAUNCHER: - case B_DATACHECKSUMSWORKER_WORKER: + case B_DATACHECKSUMWORKER_LAUNCHER: + case B_DATACHECKSUMWORKER_WORKER: return false; case B_AUTOVAC_WORKER: diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index 13a5d8e6440..49f0466040b 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -362,8 +362,8 @@ pgstat_tracks_io_bktype(BackendType bktype) case B_LOGGER: return false; - case B_DATACHECKSUMSWORKER_LAUNCHER: - case B_DATACHECKSUMSWORKER_WORKER: + case B_DATACHECKSUMWORKER_LAUNCHER: + case B_DATACHECKSUMWORKER_WORKER: case B_AUTOVAC_LAUNCHER: case B_AUTOVAC_WORKER: case B_BACKEND: diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c index 7ffc808073a..42a0e896f1d 100644 --- a/src/backend/utils/init/miscinit.c +++ b/src/backend/utils/init/miscinit.c @@ -846,7 +846,7 @@ InitializeSessionUserIdStandalone(void) */ Assert(!IsUnderPostmaster || AmAutoVacuumWorkerProcess() || AmLogicalSlotSyncWorkerProcess() || AmBackgroundWorkerProcess() || - AmDataChecksumsWorkerProcess()); + AmDataChecksumWorkerProcess()); /* call only once */ Assert(!OidIsValid(AuthenticatedUserId)); diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 2460e550f96..9acd984da42 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -921,7 +921,7 @@ InitPostgres(const char *in_dbname, Oid dboid, errhint("You should immediately run CREATE USER \"%s\" SUPERUSER;.", username != NULL ? username : "postgres"))); } - else if (AmBackgroundWorkerProcess() || AmDataChecksumsWorkerProcess()) + else if (AmBackgroundWorkerProcess() || AmDataChecksumWorkerProcess()) { if (username == NULL && !OidIsValid(useroid)) { diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 7de0a115402..bc538ac212e 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -370,8 +370,8 @@ typedef enum BackendType B_WAL_SUMMARIZER, B_WAL_WRITER, - B_DATACHECKSUMSWORKER_LAUNCHER, - B_DATACHECKSUMSWORKER_WORKER, + B_DATACHECKSUMWORKER_LAUNCHER, + B_DATACHECKSUMWORKER_WORKER, /* * Logger is not connected to shared memory and does not have a PGPROC @@ -398,9 +398,9 @@ extern PGDLLIMPORT BackendType MyBackendType; #define AmWalSummarizerProcess() (MyBackendType == B_WAL_SUMMARIZER) #define AmWalWriterProcess() (MyBackendType == B_WAL_WRITER) #define AmIoWorkerProcess() (MyBackendType == B_IO_WORKER) -#define AmDataChecksumsWorkerProcess() \ - (MyBackendType == B_DATACHECKSUMSWORKER_LAUNCHER || \ - MyBackendType == B_DATACHECKSUMSWORKER_WORKER) +#define AmDataChecksumWorkerProcess() \ + (MyBackendType == B_DATACHECKSUMWORKER_LAUNCHER || \ + MyBackendType == B_DATACHECKSUMWORKER_WORKER) #define AmSpecialWorkerProcess() \ (AmAutoVacuumLauncherProcess() || \ diff --git a/src/include/postmaster/datachecksum_state.h b/src/include/postmaster/datachecksum_state.h index 2a1ae10d55d..4426f9dc622 100644 --- a/src/include/postmaster/datachecksum_state.h +++ b/src/include/postmaster/datachecksum_state.h @@ -17,12 +17,12 @@ #include "storage/procsignal.h" -/* Possible operations the DataChecksumsWorker can perform */ -typedef enum DataChecksumsWorkerOperation +/* Possible operations the DataChecksumWorker can perform */ +typedef enum DataChecksumWorkerOperation { ENABLE_DATACHECKSUMS, DISABLE_DATACHECKSUMS, -} DataChecksumsWorkerOperation; +} DataChecksumWorkerOperation; /* * Possible states for a database entry which has been processed. Exported @@ -30,25 +30,25 @@ typedef enum DataChecksumsWorkerOperation */ typedef enum { - DATACHECKSUMSWORKER_SUCCESSFUL = 0, - DATACHECKSUMSWORKER_ABORTED, - DATACHECKSUMSWORKER_FAILED, - DATACHECKSUMSWORKER_DROPDB, -} DataChecksumsWorkerResult; + DATACHECKSUMWORKER_SUCCESSFUL = 0, + DATACHECKSUMWORKER_ABORTED, + DATACHECKSUMWORKER_FAILED, + DATACHECKSUMWORKER_DROPDB, +} DataChecksumWorkerResult; /* Prototypes for data checksum state manipulation */ -bool AbsorbDataChecksumsBarrier(ProcSignalBarrierType barrier); -void EmitAndWaitDataChecksumsBarrier(uint32 state); +bool AbsorbDataChecksumBarrier(ProcSignalBarrierType barrier); +void EmitAndWaitDataChecksumBarrier(uint32 state); /* Prototypes for data checksum background worker */ /* Start the background processes for enabling or disabling checksums */ -void StartDataChecksumsWorkerLauncher(DataChecksumsWorkerOperation op, - int cost_delay, - int cost_limit); +void StartDataChecksumWorkerLauncher(DataChecksumWorkerOperation op, + int cost_delay, + int cost_limit); /* Background worker entrypoints */ -void DataChecksumsWorkerLauncherMain(Datum arg); -void DataChecksumsWorkerMain(Datum arg); +void DataChecksumWorkerLauncherMain(Datum arg); +void DataChecksumWorkerMain(Datum arg); #endif /* DATACHECKSUM_STATE_H */ diff --git a/src/include/postmaster/proctypelist.h b/src/include/postmaster/proctypelist.h index b3477e6f17a..ec84860a613 100644 --- a/src/include/postmaster/proctypelist.h +++ b/src/include/postmaster/proctypelist.h @@ -38,8 +38,8 @@ PG_PROCTYPE(B_BACKEND, "backend", gettext_noop("client backend"), BackendMain, t PG_PROCTYPE(B_BG_WORKER, "bgworker", gettext_noop("background worker"), BackgroundWorkerMain, true) PG_PROCTYPE(B_BG_WRITER, "bgwriter", gettext_noop("background writer"), BackgroundWriterMain, true) PG_PROCTYPE(B_CHECKPOINTER, "checkpointer", gettext_noop("checkpointer"), CheckpointerMain, true) -PG_PROCTYPE(B_DATACHECKSUMSWORKER_LAUNCHER, "checksums", gettext_noop("datachecksum launcher"), NULL, false) -PG_PROCTYPE(B_DATACHECKSUMSWORKER_WORKER, "checksums", gettext_noop("datachecksum worker"), NULL, false) +PG_PROCTYPE(B_DATACHECKSUMWORKER_LAUNCHER, "checksums", gettext_noop("datachecksum launcher"), NULL, false) +PG_PROCTYPE(B_DATACHECKSUMWORKER_WORKER, "checksums", gettext_noop("datachecksum worker"), NULL, false) PG_PROCTYPE(B_DEAD_END_BACKEND, "backend", gettext_noop("dead-end client backend"), BackendMain, true) PG_PROCTYPE(B_INVALID, "postmaster", gettext_noop("unrecognized"), NULL, false) PG_PROCTYPE(B_IO_WORKER, "ioworker", gettext_noop("io worker"), IoWorkerMain, true) diff --git a/src/include/storage/subsystemlist.h b/src/include/storage/subsystemlist.h index 9ad619080be..e9bd1ce0681 100644 --- a/src/include/storage/subsystemlist.h +++ b/src/include/storage/subsystemlist.h @@ -84,7 +84,7 @@ PG_SHMEM_SUBSYSTEM(InjectionPointShmemCallbacks) #endif PG_SHMEM_SUBSYSTEM(WaitLSNShmemCallbacks) PG_SHMEM_SUBSYSTEM(LogicalDecodingCtlShmemCallbacks) -PG_SHMEM_SUBSYSTEM(DataChecksumsShmemCallbacks) +PG_SHMEM_SUBSYSTEM(DataChecksumShmemCallbacks) /* AIO subsystem. This delegates to the method-specific callbacks */ PG_SHMEM_SUBSYSTEM(AioShmemCallbacks) -- 2.47.3 ----Next_Part(Thu_May_28_12_16_22_2026_214)---- ^ permalink raw reply [nested|flat] 330+ messages in thread
end of thread, other threads:[~2026-05-26 07:56 UTC | newest] Thread overview: 330+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-23 14:09 [PATCH v7] pg_rewind: use UUIDs to detect independent same-TLI promotions Mats Kindahl <mats@kindahl.net> 2026-05-26 07:56 [PATCH 4/4] Use singular "datachecksum" consistently in process names Kyotaro Horiguchi <horikyota.ntt@gmail.com>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox